{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "54a609477af23a56",
   "metadata": {},
   "source": [
    "## Chunks（chunk shape）"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9dc0994e54811dd0",
   "metadata": {},
   "source": [
    "Dask arrays是由多个numpy arrays（or numpy-like）组成的，这些arrays的组合方式显著影响着性能，对于不同的算法，不同的组合方式，可能使这个算法执行的更快或者更慢。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d7a935de43f58aae",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-29T15:18:52.815374Z",
     "start_time": "2024-05-29T15:18:52.728712Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100 499999.5\n",
      "执行前3的如下：\n",
      "耗时：1.6262516975402832 参数配置：(100,)\n",
      "1000 499999.5\n",
      "执行前3的如下：\n",
      "耗时：0.13198375701904297 参数配置：(1000,)\n",
      "耗时：1.6262516975402832 参数配置：(100,)\n",
      "10000 499999.5\n",
      "执行前3的如下：\n",
      "耗时：0.020000934600830078 参数配置：(10000,)\n",
      "耗时：0.13198375701904297 参数配置：(1000,)\n",
      "耗时：1.6262516975402832 参数配置：(100,)\n",
      "(500000, 500000) 499999.5\n",
      "执行前3的如下：\n",
      "耗时：0.004019498825073242 参数配置：((500000, 500000),)\n",
      "耗时：0.020000934600830078 参数配置：(10000,)\n",
      "耗时：0.13198375701904297 参数配置：(1000,)\n",
      "(100000, 400000, 500000) 499999.5\n",
      "执行前3的如下：\n",
      "耗时：0.004019498825073242 参数配置：((500000, 500000),)\n",
      "耗时：0.004982709884643555 参数配置：((100000, 400000, 500000),)\n",
      "耗时：0.020000934600830078 参数配置：(10000,)\n"
     ]
    }
   ],
   "source": [
    "import dask as dd\n",
    "from dask import array as da\n",
    "import numpy as np\n",
    "\n",
    "from utils import clocked\n",
    "\n",
    "arr = np.array(range(1000000))\n",
    "\n",
    "\n",
    "@clocked()\n",
    "def compute_mean(val=100):\n",
    "    x = da.from_array(arr, chunks=(val,))\n",
    "    print(val, x.mean().compute())\n",
    "\n",
    "chunks_list = [100,1000,10000,(500000,500000),(100000,400000,500000)]\n",
    "for chunks in chunks_list:\n",
    "    compute_mean(chunks)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6258da22-919c-49c8-b8d4-6b33c20b7d35",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-29T14:14:20.428491Z",
     "start_time": "2024-05-29T14:14:20.426922Z"
    }
   },
   "source": [
    "## 指定每个chunk的形状"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "cd77119a1486c1c7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-29T15:23:47.010389Z",
     "start_time": "2024-05-29T15:23:47.005194Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100,)\n",
      "((25, 25, 25, 25),)\n",
      "(100,)\n",
      "((30, 30, 40),)\n"
     ]
    }
   ],
   "source": [
    "# 原数组为含有100个元素的一维数组\n",
    "arr2 = np.array(range(100))\n",
    "\n",
    "# 一维\n",
    "# 1.平均每块的大小 平均每块25个元素\n",
    "darr = da.from_array(arr2, chunks=(25,))\n",
    "print(darr.shape)\n",
    "print(darr.chunks)\n",
    "\n",
    "# 2.指定每一块的大小 第一块30个元素，第二块30个元素，第三块40个元素\n",
    "darr = da.from_array(arr2, chunks=(30,30,40))\n",
    "print(darr.shape)\n",
    "print(darr.chunks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "41a0ef84a35fd6c6",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-05-29T15:58:00.032353Z",
     "start_time": "2024-05-29T15:58:00.026023Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1 2 3 4 5 6]\n",
      " [7 8 9 0 1 2]\n",
      " [3 4 5 6 7 8]\n",
      " [9 0 1 2 3 4]\n",
      " [5 6 7 8 9 0]\n",
      " [1 2 3 4 5 6]]\n",
      "(6, 6)\n",
      "((3, 3), (3, 3))\n",
      "((3, 3), (2, 2, 2))\n",
      "((1, 1, 1, 1, 1, 1), (6,))\n",
      "((2, 4), (3, 3))\n",
      "((2, 2, 1, 1), (3, 2, 1))\n"
     ]
    }
   ],
   "source": [
    "# 二维\n",
    "arr3 = np.array([[1, 2, 3, 4, 5, 6],\n",
    "        [7, 8, 9, 0, 1, 2],\n",
    "        [3, 4, 5, 6, 7, 8],\n",
    "        [9, 0, 1, 2, 3, 4],\n",
    "        [5, 6, 7, 8, 9, 0],\n",
    "        [1, 2, 3, 4, 5, 6]])\n",
    "print(arr3)\n",
    "\n",
    "# 1.每个纬度（第一纬度和第二纬度）都是3 第一纬度：6/3=2 第二纬度：6/3=2  会被分成：2x2=4块 \n",
    "darr = da.from_array(arr3, chunks=3)\n",
    "print(darr.shape)\n",
    "print(darr.chunks)\n",
    "\n",
    "\n",
    "# 2.每块第一个纬度为3，第二个纬度为2   (6/3)x(6/2)=6块\n",
    "darr = da.from_array(arr3, chunks=(3,2))\n",
    "print(darr.chunks)\n",
    "\n",
    "# 3.每块第一个纬度为1，第二个纬度为6 (6/1)*(6/6)=6块\n",
    "darr = da.from_array(arr3, chunks=(1,6))\n",
    "print(darr.chunks)\n",
    "\n",
    "# 4.第一个纬度分为2和4，第二个纬度分为3和3 \n",
    "darr = da.from_array(arr3, chunks=((2,4),(3,3)))\n",
    "print(darr.chunks)\n",
    "\n",
    "# 5.第一个纬度分为2,2,1,1，第二个纬度分为3,2,1\n",
    "darr = da.from_array(arr3, chunks=((2,2,1,1),(3,2,1)))\n",
    "print(darr.chunks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7212fbfc564b6c5d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.7rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
